{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import tensorflow.contrib as tc\n",
    "import gym\n",
    "from random import sample\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    }
   ],
   "source": [
    "gamma=0.99 \n",
    "tau=0.001 \n",
    "normalize_returns=False \n",
    "normalize_observations=True\n",
    "batch_size=128 \n",
    "observation_range=(0., 1000.) \n",
    "action_range=(-0.05, 0.05) \n",
    "return_range=(-np.inf, np.inf),\n",
    "critic_l2_reg=0. \n",
    "actor_lr=1e-4 #1e-5\n",
    "critic_lr=1e-4 #3e-5\n",
    "clip_norm=None \n",
    "reward_scale=1.\n",
    "\n",
    "env = gym.make(\"Pendulum-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.]\n",
      "[-2.]\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py:3794: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Tensor(\"state_encode/masking/mul:0\", shape=(?, ?, 3), dtype=float32)\n",
      "WARNING:tensorflow:Model inputs must come from `tf.keras.Input` (thus holding past layer metadata), they cannot be the output of a previous non-Input layer. Here, a tensor specified as input to \"model\" was not an Input tensor, it was generated by layer masking.\n",
      "Note that input tensors are instantiated via `tensor = tf.keras.Input(shape)`.\n",
      "The tensor that caused the issue was: state_encode/masking/mul:0\n",
      "WARNING:tensorflow:Model inputs must come from `tf.keras.Input` (thus holding past layer metadata), they cannot be the output of a previous non-Input layer. Here, a tensor specified as input to \"model\" was not an Input tensor, it was generated by layer masking_1.\n",
      "Note that input tensors are instantiated via `tensor = tf.keras.Input(shape)`.\n",
      "The tensor that caused the issue was: state_encode/masking_1/mul:0\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Graph disconnected: cannot obtain value for tensor Tensor(\"state_encode/input_1:0\", shape=(?, ?, 3), dtype=float32) at layer \"input_1\". The following previous layers were accessed without issue: []",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-db80a13c80dd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m \u001b[0;31m#actor, critic model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mobs_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maction_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnew_action_input\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maction_output\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mQ_value_output\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m \u001b[0mactor_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mobs_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maction_input\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maction_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0mcritic_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mobs_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maction_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnew_action_input\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mQ_value_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m     \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m     \u001b[0;31m# initializing _distribution_strategy here since it is possible to call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m     \u001b[0;31m# predict on a model without compiling it.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    160\u001b[0m         'inputs' in kwargs and 'outputs' in kwargs):\n\u001b[1;32m    161\u001b[0m       \u001b[0;31m# Graph network\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_graph_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m       \u001b[0;31m# Subclassed network\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    455\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_self_setattr_tracking\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    458\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_self_setattr_tracking\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprevious_value\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py\u001b[0m in \u001b[0;36m_init_graph_network\u001b[0;34m(self, inputs, outputs, name, **kwargs)\u001b[0m\n\u001b[1;32m    313\u001b[0m     \u001b[0;31m# Keep track of the network's nodes and layers.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m     nodes, nodes_by_depth, layers, layers_by_depth = _map_graph_network(\n\u001b[0;32m--> 315\u001b[0;31m         self.inputs, self.outputs)\n\u001b[0m\u001b[1;32m    316\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_network_nodes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnodes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nodes_by_depth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnodes_by_depth\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py\u001b[0m in \u001b[0;36m_map_graph_network\u001b[0;34m(inputs, outputs)\u001b[0m\n\u001b[1;32m   1848\u001b[0m                              \u001b[0;34m'The following previous layers '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1849\u001b[0m                              \u001b[0;34m'were accessed without issue: '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1850\u001b[0;31m                              str(layers_with_complete_input))\n\u001b[0m\u001b[1;32m   1851\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput_tensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1852\u001b[0m           \u001b[0mcomputable_tensors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Graph disconnected: cannot obtain value for tensor Tensor(\"state_encode/input_1:0\", shape=(?, ?, 3), dtype=float32) at layer \"input_1\". The following previous layers were accessed without issue: []"
     ]
    }
   ],
   "source": [
    "#state place holder\n",
    "obs_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0]\n",
    "action_high = env.action_space.high\n",
    "action_low = env.action_space.low\n",
    "print(action_high)\n",
    "print(action_low)\n",
    "\n",
    "#RNN network for state\n",
    "with tf.compat.v1.variable_scope(\"state_encode\"):\n",
    "    obs_input = tf.keras.layers.Input(shape = (None,obs_dim))\n",
    "    action_input = tf.keras.layers.Input(shape = (None,action_dim))\n",
    "    mask_obs_input = tf.keras.layers.Masking(mask_value=0.0)(obs_input)\n",
    "    mask_action_input = tf.keras.layers.Masking(mask_value=0.0)(action_input)\n",
    "    #obs_h = tf.keras.layers.Dense(4,activation = 'relu')(obs_input)\n",
    "    rnn_layer,state_h = tf.keras.layers.GRU(64,return_state=True)(tf.keras.layers.Concatenate()([mask_action_input,mask_obs_input]))\n",
    "    print(obs_input)\n",
    "\n",
    "#policy network\n",
    "with tf.compat.v1.variable_scope(\"policy\"):\n",
    "    #output action\n",
    "    action_output_h = tf.keras.layers.Dense(256,activation = 'relu')(state_h)\n",
    "    action_output = tf.keras.layers.Dense(action_dim,activation = 'tanh')(action_output_h)*(action_high-action_low)/2+(action_low+action_high)/2\n",
    "    \n",
    "with tf.compat.v1.variable_scope(\"target_policy\"):\n",
    "    #output action\n",
    "    target_action_output_h = tf.keras.layers.Dense(256,activation = 'relu')(state_h)\n",
    "    target_action_output = tf.keras.layers.Dense(action_dim,activation = 'tanh')(action_output_h)*(action_high-action_low)/2+(action_low+action_high)/2\n",
    "\n",
    "    \n",
    "#output value function\n",
    "with tf.compat.v1.variable_scope(\"value_function\"):\n",
    "    value_h = tf.keras.layers.Dense(4,activation = 'relu')(state_h)\n",
    "    new_action_input = tf.keras.layers.Input(shape = (action_dim))\n",
    "    value_input = tf.keras.layers.Concatenate()([new_action_input,value_h])\n",
    "    value_h2 = tf.keras.layers.Dense(256,activation = 'relu')(value_input)\n",
    "    Q_value_output = tf.keras.layers.Dense(1,activation = 'linear')(value_h2)\n",
    "    \n",
    "with tf.compat.v1.variable_scope(\"target_value_function\"):\n",
    "    target_value_h = tf.keras.layers.Dense(4,activation = 'relu')(state_h)\n",
    "    target_new_action_input = tf.keras.layers.Input(shape = (action_dim))\n",
    "    target_value_input = tf.keras.layers.Concatenate()([target_new_action_input, target_value_h])\n",
    "    target_value_h2 = tf.keras.layers.Dense(256,activation = 'relu')(target_value_input)\n",
    "    target_Q_value_output = tf.keras.layers.Dense(1,activation = 'linear')(target_value_h2)\n",
    "\n",
    "#actor, critic model\n",
    "model = tf.keras.Model([obs_input,action_input,new_action_input],[action_output,Q_value_output])\n",
    "actor_model = tf.keras.Model([obs_input,action_input],action_output)\n",
    "critic_model = tf.keras.Model([obs_input,action_input,new_action_input],Q_value_output)\n",
    "target_actor_model = tf.keras.Model([obs_input,action_input],target_action_output)\n",
    "target_critic_model = tf.keras.Model([obs_input,action_input,target_new_action_input],target_Q_value_output)\n",
    "actor_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#computation graph\n",
    "#policy gradient ascent\n",
    "Q_grad = tf.gradients(critic_model.output,new_action_input)\n",
    "Q_action_grad = tf.compat.v1.placeholder(tf.float32, shape=(None,action_dim))\n",
    "actor_model_weights = actor_model.trainable_weights\n",
    "action_grad = tf.gradients(actor_model.output,actor_model_weights,-Q_action_grad)\n",
    "act_grad_ascent = tf.train.AdamOptimizer(actor_lr).apply_gradients(zip(action_grad,actor_model_weights))\n",
    "\n",
    "#critic mse\n",
    "adam  = tf.keras.optimizers.Adam(critic_lr)\n",
    "critic_model.compile(loss=\"mse\", optimizer=adam)\n",
    "\n",
    "#target networks\n",
    "#moving average\n",
    "ema = tf.train.ExponentialMovingAverage(decay=0.99)\n",
    "policy_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"policy\")\n",
    "value_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"value_function\")\n",
    "target_policy_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"target_policy\")\n",
    "target_value_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"target_value_function\")\n",
    "target_policy_upd = ema.apply(policy_weights)\n",
    "target_value_upd = ema.apply(value_weights)\n",
    "target_policy_assign = tf.group([tf.assign(target_policy_weights[i], ema.average(policy_weights[i])) for i in range(len(policy_weights))])\n",
    "target_value_assign = tf.group([tf.assign(target_value_weights[i], ema.average(value_weights[i])) for i in range(len(value_weights))])\n",
    "#ema.average(critic_model_weights)\n",
    "\n",
    "\n",
    "#initialization\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCH = 10000\n",
    "max_step = 200\n",
    "max_seq = 50\n",
    "batch_size = 1\n",
    "epsilon = 0.9\n",
    "epsilon_decay = 0.99995\n",
    "\n",
    "replay_buffer = []\n",
    "max_size = 10000\n",
    "rewards_history = []\n",
    "\n",
    "for epoch in range(EPOCH):\n",
    "    #print(\"epoch: \",epoch)\n",
    "    obs_seq_list = []\n",
    "    act_seq_list = []\n",
    "    obs = env.reset()\n",
    "    obs = obs.reshape((obs_dim,))\n",
    "    obs_seq_list.append(obs)\n",
    "    act_seq_list.append(np.zeros(action_dim))\n",
    "    for t in range(max_step):\n",
    "        #interaction\n",
    "        obs_seq = np.asarray(obs_seq_list)\n",
    "        act_seq = np.asarray(act_seq_list)\n",
    "        obs_seq = obs_seq.reshape((1,-1,obs_dim))\n",
    "        act_seq = act_seq.reshape((1,-1,action_dim))\n",
    "        \n",
    "        \n",
    "        #choose action\n",
    "        r = np.random.rand(1)\n",
    "        epsilon = epsilon*epsilon_decay\n",
    "        if r>epsilon:\n",
    "            action = sess.run(action_output,feed_dict={obs_input:obs_seq,action_input:act_seq})\n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "        \n",
    "        action = action.reshape((action_dim,))\n",
    "        next_obs,reward,done,_ = env.step(action)\n",
    "        next_obs = next_obs.reshape((obs_dim,))\n",
    "        \n",
    "        obs_seq_list.append(next_obs)\n",
    "        act_seq_list.append(action)\n",
    "        #keep the length same\n",
    "        if len(obs_seq_list)>max_seq:\n",
    "            obs_seq_list.pop(0)\n",
    "            act_seq_list.pop(0)\n",
    "        next_obs_seq = np.asarray(obs_seq_list)\n",
    "        next_act_seq = np.asarray(act_seq_list)\n",
    "        next_obs_seq = next_obs_seq.reshape((1,-1,obs_dim))\n",
    "        next_act_seq = next_act_seq.reshape((1,-1,action_dim))\n",
    "        \n",
    "        #pad data\n",
    "        pad_width = max_seq-np.size(obs_seq,1)\n",
    "        next_pad_width = max_seq-np.size(next_obs_seq,1)\n",
    "        \n",
    "        #record\n",
    "        replay_buffer.append([obs_seq,act_seq,action,reward,next_obs_seq,next_act_seq,done])\n",
    "        if len(replay_buffer)>max_size:\n",
    "            replay_buffer.pop(0)\n",
    "        \n",
    "        #move to the next\n",
    "        obs = next_obs\n",
    "        \n",
    "        \n",
    "        #train\n",
    "        #get data from the buffer\n",
    "        if len(replay_buffer)>batch_size and t!=0:\n",
    "            train_datas = sample(replay_buffer,batch_size)\n",
    "            \n",
    "            for train_data in train_datas:\n",
    "            \n",
    "                obs_seq_data = train_data[0]\n",
    "                act_seq_data = train_data[1]\n",
    "                action_data = train_data[2]\n",
    "                reward_data = train_data[3]\n",
    "                next_obs_data = train_data[4]\n",
    "                next_act_data = train_data[5]\n",
    "                done_data = train_data[6]\n",
    "\n",
    "                #train actor\n",
    "                new_action = sess.run(action_output, feed_dict={\n",
    "                    obs_input:  obs_seq_data,\n",
    "                    action_input: act_seq_data\n",
    "                })\n",
    "                Q_action_grads = sess.run(Q_grad, feed_dict={\n",
    "                    obs_input:  obs_seq_data,\n",
    "                    action_input: act_seq_data,\n",
    "                    new_action_input: new_action\n",
    "                })[0]\n",
    "                sess.run(act_grad_ascent, feed_dict={\n",
    "                    obs_input:  obs_seq_data,\n",
    "                    action_input: act_seq_data,\n",
    "                    Q_action_grad:Q_action_grads\n",
    "                })\n",
    "                #train critic\n",
    "                target_actions = sess.run(target_action_output, feed_dict = {\n",
    "                    obs_input: next_obs_data,\n",
    "                    action_input: next_act_data,\n",
    "                })\n",
    "                target_Qs = sess.run(target_Q_value_output, feed_dict = {\n",
    "                    obs_input: next_obs_data,\n",
    "                    action_input: next_act_data,\n",
    "                    target_new_action_input: target_actions\n",
    "                })\n",
    "                Q_y = reward_data + gamma*target_Qs*(1-done)\n",
    "                evaluation = critic_model.fit([obs_seq_data,act_seq_data,action_data], Q_y, verbose=0)\n",
    "    #             if t==max_step-5:\n",
    "    #                 print(t,evaluation.history)\n",
    "                #update target\n",
    "                sess.run(target_policy_upd)\n",
    "                sess.run(target_value_upd)\n",
    "                sess.run(target_policy_assign)\n",
    "                sess.run(target_value_assign)\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "                \n",
    "    #evaluate\n",
    "    if epoch%10==0:\n",
    "        obs_seq_list = []\n",
    "        act_seq_list = []\n",
    "        ave_rewards = 0\n",
    "        for times in range(5):\n",
    "            rewards = 0\n",
    "            obs = env.reset()          \n",
    "            for t in range(500):\n",
    "                #interaction\n",
    "                obs_seq = np.asarray(obs_seq_list)\n",
    "                act_seq = np.asarray(act_seq_list)\n",
    "                obs_seq = obs_seq.reshape((1,-1,obs_dim))\n",
    "                act_seq = act_seq.reshape((1,-1,action_dim))\n",
    "                #choose action\n",
    "                action = sess.run(action_output,feed_dict={obs_input:obs_seq,action_input:act_seq})\n",
    "                action = action.reshape((action_dim,))\n",
    "                next_obs,reward,done,_ = env.step(action)\n",
    "                obs = next_obs.reshape((obs_dim,))\n",
    "                obs_seq_list.append(obs)   \n",
    "                act_seq_list.append(action)\n",
    "                if len(obs_seq_list)>max_seq:\n",
    "                    obs_seq_list.pop(0)\n",
    "                    act_seq_list.pop(0)\n",
    "\n",
    "                rewards = rewards+reward\n",
    "                if done:\n",
    "                    break\n",
    "            ave_rewards = ave_rewards+rewards\n",
    "            \n",
    "        ave_rewards = ave_rewards/5\n",
    "        rewards_history.append(ave_rewards)\n",
    "        print(\"epoch: \",epoch)\n",
    "        print(\"rewards: \", ave_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [0,1,2,3]\n",
    "a.append(4)\n",
    "print(a)\n",
    "a.pop(0)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
